# -*- coding: utf-8 -*-
import scrapy
from tutorial.settings import logger
from pyquery import PyQuery as pq
import json, time
from tutorial.items import QuotesItem

class QuotesSpider(scrapy.Spider):
    name = 'quotes'
    allowed_domains = ['quotes.toscrape.com']
    start_urls = ['http://quotes.toscrape.com/']

    def parse(self, response):
        # result = json.loads(response.text)
        logger.info('myprint')
        html = response.text
        doc = pq(html)
        items = doc('.row .quote').items()
        for obj in items:
            item = QuotesItem()
            item['author'] = obj.find('.author').text()
            item['tags'] = obj.find('.tags .tag').text()
            item['text'] = obj.find('.text').text()
            yield item
            
        time.sleep(1)
        next = doc('.pager').find('.next a').attr('href')
        logger.info('页数，第' + next[-2:-1] + '页')
        url = response.urljoin(next)
        yield scrapy.Request(url=url, callback=self.parse)
